{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN Preprocessing \n",
    "\n",
    "Preprocessing, stacking and CNN architecture are based on the [DQN-Paper](https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf)\n",
    "\n",
    "Preprocessing of the DQN CNN:\n",
    "- 1. grayscaling the image to a shape (1, 210, 160)\n",
    "- 2. Resizing the image to 110x84\n",
    "- 3. cropping the image to a shape of (1, 84, 84)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(\"Pong-v0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(210, 160, 3)\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "print(state.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAM4AAAD8CAYAAAA/rZtiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOX0lEQVR4nO3dfYwc9X3H8fenNhiXB/HsInCKQSYqVK1DLIqEQLQ0AawqDpVIjSripCgHEkiJlEoxILWoUqSUhiClD0QgrEBFDbSEwB9OwbKSoEg1sU2MsWMMNnHgsHtOnApIeUjP/vaP+V2znHe59Xd2b2e3n5d02t3fzNx8R8eHefDMdxURmNmR+Y1BF2A2jBwcswQHxyzBwTFLcHDMEhwcs4S+BUfSVZJ2StolaVW/1mM2COrHv+NImgO8BHwMGAc2AtdFxI97vjKzAejXHuciYFdEvBIRvwIeBpb3aV1ms25un37vmcBrLZ/HgT/oNLOkD9ztnb7gmB6VZda9/RPv/jwiTms3rV/BUZux94VD0hgwBnD8CUfxmZvO61MpOZ+7/IIjXua+723vQyXD7933njriZY6Zd2UfKjkyf3/n9p92mtavQ7VxYGHL57OAva0zRMS9EbE0IpbOnz+nT2WY9Ue/grMRWCxpkaSjgRXAk31al9ms68uhWkRMSroFeAqYA6yOCB/H2Mjo1zkOEbEWWNuv3z/b2p2/ZM6DrP35S+Y8aJB854BZgoNjluDgmCU4OGYJDo5ZgoNjluDgmCU4OGYJDo5ZgoNjluDgmCU4OGYJfbvJc9T4hs7eGbYbOtvxHscswcExS3BwzBJ8jtOBG2/0ThMab/Raeo8jaaGk70raIWm7pM+X8TskvS5pS/lZ1rtyzZqhzh5nEvhiRDwn6Xhgs6R1ZdrdEfHV+uWZNVM6OBGxD9hX3r8laQdVI8Ij9svJSTZMHMiWYjbrenJxQNLZwEeAZ8vQLZK2Slot6aRerMOsSWoHR9JxwGPAFyLiTeAe4FxgCdUe6a4Oy41J2iRp0+S7h+qWYTaragVH0lFUoXkoIr4FEBETEXEwIg4B91E1YD9MayfPucf4qrgNlzpX1QTcD+yIiK+1jJ/RMts1wLZ8eWbNVOeq2iXA9cALkraUsduA6yQtoWqyvge4sVaFZg1U56raD2j/rQQj073TrBOfXJglODhmCQ6OWUIjbvI8bu5cLl5wyqDLMHufjfxnx2ne45glODhmCQ6OWYKDY5bg4JglODhmCQ6OWYKDY5bg4JglODhmCQ6OWUKjgrNh4oC73dhQaFRwzIZF7bujJe0B3gIOApMRsVTSycAjwNlUj09/KiL+q+66zJqiV3ucP4yIJRGxtHxeBayPiMXA+vLZbGT063mc5cDl5f0DwPeAL820kJ/JsWHRiz1OAE9L2ixprIwtKC1yp1rlnt6D9Zg1Ri/2OJdExF5JpwPrJL3YzUIlZGMAx59wVA/KMJs9tfc4EbG3vO4HHqfq3Dkx1ZiwvO5vs9z/dfKcP39O3TLMZlXdFrjHlq/4QNKxwMepOnc+Cawss60EnqizHrOmqXuotgB4vOqGy1zgXyLi3yVtBB6VdAPwKnBtzfWYNUqt4ETEK8Dvtxk/AFxR53ebNZnvHDBLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLSD8BKunDVN06p5wD/BVwIvA54Gdl/LaIWJuu0KyB0sGJiJ3AEgBJc4DXqbrcfBa4OyK+2pMKzRqoV4dqVwC7I+KnPfp9Zo3Wq+CsANa0fL5F0lZJqyWd1KN1mDVG7eBIOhr4BPCvZege4Fyqw7h9wF0dlhuTtEnSpnfeOVi3DLNZ1Ys9ztXAcxExARARExFxMCIOAfdRdfY8jDt52jDrRXCuo+Uwbar1bXENVWdPs5FSqyGhpN8EPgbc2DJ8p6QlVN9isGfaNLORULeT59vAKdPGrq9VkdkQ8J0DZgkOjlmCg2OW4OCYJTg4ZgkOjlmCg2OW4OCYJTg4ZgkOjlmCg2OW4OCYJTg4ZgkOjlmCg2OW4OCYJdR6kM2sKd5976n3fT5m3pV9XV9Xe5zS5mm/pG0tYydLWifp5fJ6UhmXpK9L2lVaRF3Yr+LNBqXbQ7VvAldNG1sFrI+IxcD68hmqrjeLy88YVbsos5HSVXAi4hngF9OGlwMPlPcPAJ9sGX8wKhuAE6d1vjEbenUuDiyIiH0A5fX0Mn4m8FrLfONl7H3ckNCGWT+uqqnNWBw24IaENsTqBGdi6hCsvO4v4+PAwpb5zgL21liPWePUCc6TwMryfiXwRMv4p8vVtYuBN6YO6cxGRVf/jiNpDXA5cKqkceCvga8Aj0q6AXgVuLbMvhZYBuwC3qb6vhyzkdJVcCLiug6TrmgzbwA31ynKrOl8y41ZgoNjluDgmCU4OGYJDo5ZgoNjluDncWwk9Pv5m+m8xzFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEmYMTocunn8n6cXSqfNxSSeW8bMlvSNpS/n5Rj+LNxuUbvY43+TwLp7rgN+NiN8DXgJubZm2OyKWlJ+belOmWbPMGJx2XTwj4umImCwfN1C1gDL7f6MX5zh/AXyn5fMiST+S9H1Jl3ZayJ08bZjVeqxA0u3AJPBQGdoHfCgiDkj6KPBtSRdExJvTl42Ie4F7ARb81vzDOn2aNVl6jyNpJfAnwJ+XllBExHsRcaC83wzsBs7rRaFmTZIKjqSrgC8Bn4iIt1vGT5M0p7w/h+qrPl7pRaFmTTLjoVqHLp63AvOAdZIANpQraJcBfyNpEjgI3BQR078exGzozRicDl087+8w72PAY3WLMms63zlgluDgmCU4OGYJDo5ZgoNjluDgmCU4OGYJDo5ZgoNjluDgmCU4OGYJDo5ZgoNjluDgmCU4OGYJDo5ZgoNjlpDt5HmHpNdbOnYua5l2q6RdknZKmt1vNDWbJdlOngB3t3TsXAsg6XxgBXBBWeafppp3mI2SVCfPD7AceLi0ifoJsAu4qEZ9Zo1U5xznltJ0fbWkk8rYmcBrLfOMl7HDuJOnDbNscO4BzgWWUHXvvKuMq828bbt0RsS9EbE0IpbOn++jORsuqeBExEREHIyIQ8B9/PpwbBxY2DLrWcDeeiWaNU+2k+cZLR+vAaauuD0JrJA0T9Iiqk6eP6xXolnzZDt5Xi5pCdVh2B7gRoCI2C7pUeDHVM3Yb44In8DYyOlpJ88y/5eBL9cpyqzpfOeAWYKDY5bg4JglODhmCQ6OWYKDY5bg4JglODhmCQ6OWYKDY5bg4JglODhmCQ6OWYKDY5bg4JglODhmCdmGhI+0NCPcI2lLGT9b0jst077Rz+LNBmXGJ0CpGhL+A/Dg1EBE/NnUe0l3AW+0zL87Ipb0qkCzJurm0elnJJ3dbpokAZ8C/qi3ZZk1W91znEuBiYh4uWVskaQfSfq+pEtr/n6zRurmUO2DXAesafm8D/hQRByQ9FHg25IuiIg3py8oaQwYAzj+hKNqlmE2u9J7HElzgT8FHpkaKz2jD5T3m4HdwHntlncnTxtmdQ7V/hh4MSLGpwYknTb17QSSzqFqSPhKvRLNmqeby9FrgP8APixpXNINZdIK3n+YBnAZsFXS88C/ATdFRLffdGA2NLINCYmIz7QZewx4rH5ZZs3mOwfMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcs4S6j073xC8nJ9kwcWDQZZh1zXscswQHxyyhm0enF0r6rqQdkrZL+nwZP1nSOkkvl9eTyrgkfV3SLklbJV3Y740wm23d7HEmgS9GxO8AFwM3SzofWAWsj4jFwPryGeBqqiYdi6naP93T86rNBmzG4ETEvoh4rrx/C9gBnAksBx4osz0AfLK8Xw48GJUNwImSzuh55WYDdETnOKUV7keAZ4EFEbEPqnABp5fZzgRea1lsvIyZjYyugyPpOKoONl9o15mzddY2Y9Hm941J2iRp0+S7h7otw6wRugqOpKOoQvNQRHyrDE9MHYKV1/1lfBxY2LL4WcDe6b+ztZPn3GN8cc+GSzdX1QTcD+yIiK+1THoSWFnerwSeaBn/dLm6djHwxtQhndmo6ObOgUuA64EXpr5ACrgN+ArwaOns+SpwbZm2FlgG7ALeBj7b04rNGqCbTp4/oP15C8AVbeYP4OaadZk1mk8uzBIcHLMEB8cswcExS3BwzBJUXQQbcBHSz4D/Bn4+6Fp66FRGZ3tGaVug++357Yg4rd2ERgQHQNKmiFg66Dp6ZZS2Z5S2BXqzPT5UM0twcMwSmhScewddQI+N0vaM0rZAD7anMec4ZsOkSXscs6Ex8OBIukrSztLcY9XMSzSPpD2SXpC0RdKmMta2mUkTSVotab+kbS1jQ9uMpcP23CHp9fI32iJpWcu0W8v27JR0ZVcriYiB/QBzgN3AOcDRwPPA+YOsKbkde4BTp43dCawq71cBfzvoOj+g/suAC4FtM9VP9cjId6jumL8YeHbQ9Xe5PXcAf9lm3vPLf3fzgEXlv8c5M61j0Huci4BdEfFKRPwKeJiq2cco6NTMpHEi4hngF9OGh7YZS4ft6WQ58HBEvBcRP6F6juyimRYadHBGpbFHAE9L2ixprIx1amYyLEaxGcst5fBydcuhc2p7Bh2crhp7DIFLIuJCqp5yN0u6bNAF9dGw/s3uAc4FlgD7gLvKeGp7Bh2crhp7NF1E7C2v+4HHqXb1nZqZDItazViaJiImIuJgRBwC7uPXh2Op7Rl0cDYCiyUtknQ0sIKq2cfQkHSspOOn3gMfB7bRuZnJsBipZizTzsOuofobQbU9KyTNk7SIqgPtD2f8hQ24ArIMeInqasbtg64nUf85VFdlnge2T20DcApVa+CXy+vJg671A7ZhDdXhy/9Q/R/4hk71Ux3a/GP5e70ALB10/V1uzz+XereWsJzRMv/tZXt2Ald3sw7fOWCWMOhDNbOh5OCYJTg4ZgkOjlmCg2OW4OCYJTg4ZgkOjlnC/wKhLjBuvkWsmwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(state)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grayscaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAM4AAAD8CAYAAAA/rZtiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOXUlEQVR4nO3df4wc9XnH8fenJiCXiwQXA0LGgEFOVKjaC7EoEgLR0iRgVXGoRGKrIm6KekYCKZFSKQakFlWKSNMQpKgtkRFWoKL8aB2C/3AaLCsNilQIhhwGYgy245jDlh2Tih/llNTm6R/zvWY57/nWz+zezm4/L+m0u9+ZuXlGx4f54ZlnFRGY2Yn5rX4XYDaIHByzBAfHLMHBMUtwcMwSHByzhJ4FR9I1knZK2iVpXa/WY9YP6sW/40haALwCfByYBJ4BVkfET7u+MrM+6NUe51JgV0TsiYhfAw8DK3u0LrN5d1KPfu9i4LWWz5PAH8w2s6Tj7vbOPffcLpVl1rl9+/Ydjogz2k3rVXDUZux94ZA0DowDjI6Ocuedd/aolJzx8fETXmb9+vU9qGTwLVq06ISXOXz4cA8qOTFr1679+WzTenWoNgksafl8DrC/dYaIWB8RyyNi+cjISI/KMOuNXgXnGWCZpKWSTgZWAZt6tC6zedeTQ7WIOCLpFuD7wAJgQ0S81It1mfVDr85xiIjNwOZe/f751u78JXMeZO3PXzLnQf3kOwfMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFL6NlNnsPGN3R2z6Dd0NmO9zhmCQ6OWYKDY5bgc5xZuPFG9zSh8Ua3pfc4kpZI+oGkHZJekvSFMn6HpNclTZSfFd0r16wZ6uxxjgBfiojnJH0QeFbSljLt7oj4ev3yzJopHZyIOAAcKO/flrSDqhHhCZuammJiYiJbitm868rFAUnnAx8Fni5Dt0jaLmmDpNO7sQ6zJqkdHEkjwEbgixHxFnAPcCEwRrVHumuW5cYlbZO0bWpqqm4ZZvOqVnAkfYAqNA9GxHcAIuJgRByNiPeAe6kasB+jtZPnwoUL65RhNu/qXFUTcB+wIyK+0TJ+dsts1wEv5ssza6Y6V9UuB24AXpA0fWZ/G7Ba0hhVk/W9wNpaFZo1UJ2raj+i/bcSDE33TrPZ+JYbswQHxyzBwTFLaMRNngsXLmRsbKzfZZh1zHscswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLaFRwJiYm3O3GBkKjgmM2KGrfHS1pL/A2cBQ4EhHLJY0CjwDnUz0+/ZmI+K+66zJrim7tcf4wIsYiYnn5vA7YGhHLgK3ls9nQ6NXzOCuBq8r7+4H/AL4810J+JscGRTf2OAE8IelZSdNfW3ZWaZE73Sr3zC6sx6wxurHHuTwi9ks6E9gi6eVOFiohGwcYHR3tQhlm86f2Hici9pfXQ8BjVJ07D043Jiyvh9os93+dPEdGRuqWYTav6rbAPbV8xQeSTgU+QdW5cxOwpsy2Bni8znrMmqbuodpZwGNVN1xOAv4lIv5d0jPAo5JuBPYB19dcj1mj1ApOROwBfr/N+BvA1XV+t1mT+c4BswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcs4T0E6CSPkLVrXPaBcBfA6cBfwn8oozfFhGb0xWaNVA6OBGxExgDkLQAeJ2qy83ngbsj4utdqdCsgbp1qHY1sDsift6l32fWaN0KzirgoZbPt0jaLmmDpNO7tA6zxqgdHEknA58C/rUM3QNcSHUYdwC4a5blxiVtk7TtnXfeqVuG2bzqxh7nWuC5iDgIEBEHI+JoRLwH3EvV2fMY7uRpg6wbwVlNy2HadOvb4jqqzp5mQ6VWQ0JJvw18HFjbMvw1SWNU32Kwd8Y0s6FQt5Pnu8CHZozdUKsiswHgOwfMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEhwcswQHxyzBwTFLqPUgm1lTLFq06H2fDx8+3NP1dbTHKW2eDkl6sWVsVNIWSa+W19PLuCR9U9Ku0iLqkl4Vb9YvnR6qfRu4ZsbYOmBrRCwDtpbPUHW9WVZ+xqnaRZkNlY6CExFPAr+cMbwSuL+8vx/4dMv4A1F5CjhtRucbs4FX5+LAWRFxAKC8nlnGFwOvtcw3Wcbexw0JbZD14qqa2ozFMQNuSGgDrE5wDk4fgpXXQ2V8EljSMt85wP4a6zFrnDrB2QSsKe/XAI+3jH+uXF27DHhz+pDObFh09O84kh4CrgIWSZoE/gb4KvCopBuBfcD1ZfbNwApgF/Au1fflmA2VjoITEatnmXR1m3kDuLlOUWZN51tuzBIcHLMEB8cswcExS3BwzBIcHLMEP49jQ6HXz9/M5D2OWYKDY5bg4JglODhmCQ6OWYKDY5bg4JglODhmCQ6OWYKDY5YwZ3Bm6eL595JeLp06H5N0Whk/X9KUpIny861eFm/WL53scb7NsV08twC/GxG/B7wC3NoybXdEjJWfm7pTplmzzBmcdl08I+KJiDhSPj5F1QLK7P+Nbpzj/AXwvZbPSyX9RNIPJV0x20Lu5GmDrNZjBZJuB44AD5ahA8C5EfGGpI8B35V0cUS8NXPZiFgPrAc477zzjun0adZk6T2OpDXAnwB/VlpCERG/iog3yvtngd3Ah7tRqFmTpIIj6Rrgy8CnIuLdlvEzJC0o7y+g+qqPPd0o1KxJ5jxUm6WL563AKcAWSQBPlStoVwJ/K+kIcBS4KSJmfj2I2cCbMzizdPG8b5Z5NwIb6xZl1nS+c8AswcExS3BwzBIcHLMEB8cswcExS3BwzBIcHLMEB8cswcExS3BwzBIcHLMEB8cswcExS3BwzBIcHLMEB8csIdvJ8w5Jr7d07FzRMu1WSbsk7ZT0yV4VbtZP2U6eAHe3dOzcDCDpImAVcHFZ5p+mm3eYDZNUJ8/jWAk8XNpE/QzYBVxaoz6zRqpzjnNLabq+QdLpZWwx8FrLPJNl7Bju5GmDLBuce4ALgTGq7p13lXG1mbdtl86IWB8RyyNi+cjISLIMs/5IBSciDkbE0Yh4D7iX3xyOTQJLWmY9B9hfr0Sz5sl28jy75eN1wPQVt03AKkmnSFpK1cnzx/VKNGuebCfPqySNUR2G7QXWAkTES5IeBX5K1Yz95og42pvSzfqnq508y/xfAb5SpyizpvOdA2YJDo5ZgoNjluDgmCU4OGYJDo5ZgoNjluDgmCU4OGYJDo5ZgoNjluDgmCU4OGYJDo5ZgoNjluDgmCVkGxI+0tKMcK+kiTJ+vqSplmnf6mXxZv0y5xOgVA0J/wF4YHogIj47/V7SXcCbLfPvjoixbhVo1kSdPDr9pKTz202TJOAzwB91tyyzZqt7jnMFcDAiXm0ZWyrpJ5J+KOmKmr/frJE6OVQ7ntXAQy2fDwDnRsQbkj4GfFfSxRHx1swFJY0D4wCjo6M1yzCbX+k9jqSTgD8FHpkeKz2j3yjvnwV2Ax9ut7w7edogq3Oo9sfAyxExOT0g6YzpbyeQdAFVQ8I99Uo0a55OLkc/BPwn8BFJk5JuLJNW8f7DNIArge2Sngf+DbgpIjr9pgOzgZFtSEhE/HmbsY3AxvplmTWb7xwwS3BwzBIcHLMEB8cswcExS3BwzBIcHLMEB8cswcExS3BwzBIcHLMEB8cswcExS3BwzBLqPjrdFVNTU0xMTPS7DLOOeY9jluDgmCV08uj0Ekk/kLRD0kuSvlDGRyVtkfRqeT29jEvSNyXtkrRd0iW93giz+dbJHucI8KWI+B3gMuBmSRcB64CtEbEM2Fo+A1xL1aRjGVX7p3u6XrVZn80ZnIg4EBHPlfdvAzuAxcBK4P4y2/3Ap8v7lcADUXkKOE3S2V2v3KyPTugcp7TC/SjwNHBWRByAKlzAmWW2xcBrLYtNljGzodFxcCSNUHWw+WK7zpyts7YZiza/b1zSNknbpqamOi3DrBE6Co6kD1CF5sGI+E4ZPjh9CFZeD5XxSWBJy+LnAPtn/s7WTp4LFy7M1m/WF51cVRNwH7AjIr7RMmkTsKa8XwM83jL+uXJ17TLgzelDOrNh0cmdA5cDNwAvTH+BFHAb8FXg0dLZcx9wfZm2GVgB7ALeBT7f1YrNGqCTTp4/ov15C8DVbeYP4OaadZk1mu8cMEtwcMwSHByzBAfHLMHBMUtQdRGsz0VIvwD+Gzjc71q6aBHDsz3DtC3Q+facFxFntJvQiOAASNoWEcv7XUe3DNP2DNO2QHe2x4dqZgkOjllCk4Kzvt8FdNkwbc8wbQt0YXsac45jNkiatMcxGxh9D46kayTtLM091s29RPNI2ivpBUkTkraVsbbNTJpI0gZJhyS92DI2sM1YZtmeOyS9Xv5GE5JWtEy7tWzPTkmf7GglEdG3H2ABsBu4ADgZeB64qJ81JbdjL7BoxtjXgHXl/Trg7/pd53HqvxK4BHhxrvqpHhn5HtUd85cBT/e7/g635w7gr9rMe1H57+4UYGn573HBXOvo9x7nUmBXROyJiF8DD1M1+xgGszUzaZyIeBL45YzhgW3GMsv2zGYl8HBE/Coifkb1HNmlcy3U7+AMS2OPAJ6Q9Kyk8TI2WzOTQTGMzVhuKYeXG1oOnVPb0+/gdNTYYwBcHhGXUPWUu1nSlf0uqIcG9W92D3AhMAYcAO4q46nt6XdwOmrs0XQRsb+8HgIeo9rVz9bMZFDUasbSNBFxMCKORsR7wL385nAstT39Ds4zwDJJSyWdDKyiavYxMCSdKumD0++BTwAvMnszk0ExVM1YZpyHXUf1N4Jqe1ZJOkXSUqoOtD+e8xc24ArICuAVqqsZt/e7nkT9F1BdlXkeeGl6G4APUbUGfrW8jva71uNsw0NUhy//Q/V/4Btnq5/q0OYfy9/rBWB5v+vvcHv+udS7vYTl7Jb5by/bsxO4tpN1+M4Bs4R+H6qZDSQHxyzBwTFLcHDMEhwcswQHxyzBwTFLcHDMEv4XEyAxNcsPr2sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "gray = cv2.cvtColor(state, cv2.COLOR_RGB2GRAY)\n",
    "plt.imshow(gray, cmap=\"gray\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAM4AAAD7CAYAAAAiqKvaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAM3klEQVR4nO3dX4wd5XnH8e+DHWIDNn9r49ioGMkiQUgpyBCSVFUwiUopirmACooqlFryDWlIiJRAexH1DqQISKUKYUFSt7L4U4IKQihRZIiqXtTFBCsBHAcXIlhsMAiDrXgtsubpxZlV186u/zyzZ2d29f1Iq7PzznvOPBr7p/fM2dF5IjORdGJO6roAaTYyOFKBwZEKDI5UYHCkAoMjFQwlOBFxdUTsiIidEXHHMI4hdSmm++84ETEP+A3wFWAEeB64KTNfmdYDSR2aP4TXvBzYmZmvAUTEI8BaYMrgRMRR07tkyZJpLfB4LV68+ITm79u3b0iVzH7z5s07ofmHDh0aUiXHb9++fYyOjsZk+4YRnOXAmxO2R4DPHTkpItYD64/nBW+++ebpqewErVmz5oTmP/vss0OqZPZbtGjRCc3fv3//kCo5fps2bZpy3zCCM1lC/2BFycwNwAaApUuXZlfhOJprr732hOYbnKmdfvrpJzS/D8E5mmF8ODACnDdhewWwawjHkTozjBXneWBVRKwE3gJuBP56CMeZcTt27Dhs+8ILL+yoktnv4MGDh20vWLCgo0pqpj04mTkWEV8HfgrMA36YmS9P93GkLg1jxSEznwGeGcZrd+mBBx44bPuee+7pqJLZ77333jtse8WKFR1VUuOdA1KBwZEKDI5UYHCkAoMjFRgcqcDgSAUGRyowOFKBwZEKDI5UMJR71eYq702bPrPt3rQjueJIBQZHKjA4UoHXOEdx++23d13CnDEyMtJ1CdPKFUcq6MWKMzY2xttvv911GdJhxsbGptzniiMV9GLFGR0dZfv27V2XIR1mdHR0yn2uOFKBwZEKDI5UYHCkAoMjFRgcqcDgSAUGRyowOFJBL+4cWLRoEVdeeWXXZUiH2bVr6n5orjhSQTk4EXFeRDwXEdsj4uWIuK0ZPysifhYRrzaPZ05fuVI/tFlxxoBvZ+ZngCuAWyPiIuAOYHNmrgI2N9vSnFIOTmbuzsxfNL/vB7YzaNW+FtjYTNsIXNe2SKlvpuUaJyLOBy4BtgBLM3M3DMIFLJniOesjYmtEbD3a7dtSH7UOTkScBvwY+GZm7jve52XmhsxcnZmrFy5c2LYMaUa1Ck5EfIJBaDZl5hPN8DsRsazZvwzY065EqX/afKoWwEPA9syc+BWXTwG3NL/fAjxZL0/qpzZ/AP0i8DfAryJiWzP298BdwGMRsQ54A7ihXYlS/5SDk5n/BcQUu6+qvOb4N92ce+65xaqkmeGdA1JBL+5VGzf+TTeuOOo7VxypoFcrzmmnndZ1CdJxccWRCnq14lx22WVdlyAdF1ccqcDgSAUGRyowOFKBwZEKDI5UYHCkAoMjFRgcqcDgSAUGRyowOFKBwZEKDI5UYHCkAoMjFRgcqcDgSAUGRyowOFKBwZEKDI5UYHCkAoMjFUxHK8N5EfFiRDzdbK+MiC1Nu/ZHI+Lk9mVK/TIdK85tDDpOj7sbuLdp174XWDcNx5B6pW0P0BXAXwIPNtsBrAEeb6bYrl1zUtsV5z7gO8DHzfbZwAeZOdZsjwDLJ3ui7do1m7VpnnstsCczX5g4PMnUnOz5tmvXbNa2ee5XI+IaYAGwmMEKdEZEzG9WnRXArvZlSv1SXnEy887MXJGZ5wM3As9m5s3Ac8D1zTTbtWtOGsbfcb4L3B4ROxlc8zw0hGNInZqWxlKZ+XPg583vrwGXT8frSn3lnQNSgcGRCgyOVGBwpAKDIxUYHKnA4EgFBkcqMDhSgcGRCgyOVGBwpAKDIxVMy93RUtcWLVp02OPBgwcBeP/994dyPFccqcAVR3PCSSeddNjj0I83I0eR5hhXHM1Jw155XHGkAlcczUkff/zxsSe14IojFbjiaE7yGkfqIYMjFRgcqcBrHM1Jfqom9ZArjuYkP1WTesgVR3PC+DXNRx99BMDY2NjRprfmiiMVtFpxIuIMBh2nL2bQ6/NvgR3Ao8D5wG+Bv8rMva2qlI5h//79hz0OW9sV5wfATzLz08Bnge3AHcDmzFwFbG62pTmlTdfpxcCf0bQqzMyPMvMDYC2wsZm2EbiubZFS37RZcS4A3gV+FBEvRsSDEXEqsDQzdwM0j0sme3JErI+IrRGxdXR0tEUZ0sxrE5z5wKXA/Zl5CfA7TuBtWWZuyMzVmbl64cKFLcqQZl6b4IwAI5m5pdl+nEGQ3omIZQDN4552JUr9Uw5OZr4NvBkRFzZDVwGvAE8BtzRjtwBPtqpQ6qG2fwD9O2BTRJwMvAZ8jUEYH4uIdcAbwA0tjyH1TqvgZOY2YPUku65q87pS33nngFRgcKQCgyMVGBypwOBIBQZHKjA4UoHBkQoMjlRgcKQCgyMVGBypwOBIBQZHKjA4UoHBkQoMjlRgcKQCgyMVGBypwOBIBQZHKjA4UoHBkQoMjlRgcKQCgyMVGBypwOBIBQZHKmgVnIj4VkS8HBEvRcTDEbEgIlZGxJaIeDUiHm1650hzSpuu08uBbwCrM/NiYB5wI3A3cG/Trn0vsG46CpX6pO1btfnAwoiYD5wC7AbWMOgHCrZr1xzVpgfoW8D3GbQr3A18CLwAfJCZY820EWD5ZM+3XbtmszZv1c4E1gIrgU8BpwJ/McnUnOz5tmvXbNbmrdqXgdcz893M/D3wBPAF4IzmrRvACmBXyxql3mkTnDeAKyLilIgI/r9d+3PA9c0c27VrTmpzjbOFwYcAvwB+1bzWBuC7wO0RsRM4G3hoGuqUeqVtu/bvAd87Yvg14PI2ryv1nXcOSAUGRyowOFKBwZEKDI5UYHCkAoMjFRgcqcDgSAUGRyowOFKBwZEKDI5UYHCkAoMjFRgcqcDgSAUGRyowOFKBwZEKDI5UYHCkAoMjFRgcqcDgSAUGRyowOFKBwZEKDI5UYHCkAoMjFRwzOBHxw4jYExEvTRg7KyJ+FhGvNo9nNuMREf8UETsj4pcRcekwi5e6cjwrzr8AVx8xdgewOTNXAZubbRg0z13V/KwH7p+eMqV+OWZwMvM/gfePGF4LbGx+3whcN2H8X3Pgvxk00l02XcVKfVG9xlmambsBmsclzfhy4M0J80aasT8QEesjYmtEbB0dHS2WIXWjVQ/QScQkYznZxMzcwKDZLuecc07u3bt3mkuR2jl06NCU+6orzjvjb8Gaxz3N+Ahw3oR5K4BdxWNIvVVdcZ4CbgHuah6fnDD+9Yh4BPgc8OH4W7qjOXDgANu2bSuWIg3HgQMHptx3zOBExMPAl4BzImKEQXv2u4DHImId8AZwQzP9GeAaYCdwAPham8KlvjpmcDLzpil2XTXJ3ARubVuU1HfeOSAVGBypwOBIBQZHKjA4UoHBkQoMjlRgcKQCgyMVGBypwOBIBQZHKjA4UkEMbmjuuIiId4HfAe91XctRnEN/6+tzbdDv+o5W2x9n5h9NtqMXwQGIiK2ZubrrOqbS5/r6XBv0u75qbb5VkwoMjlTQp+Bs6LqAY+hzfX2uDfpdX6m23lzjSLNJn1YcadYwOFJBL4ITEVdHxI6my8Edx37GUGs5LyKei4jtEfFyRNzWjE/aoaGjGudFxIsR8XSzvTIitjS1PRoRJ3dY2xkR8XhE/Lo5h5/v2bn7VvPv+lJEPBwRCyrnr/PgRMQ84J8ZdDq4CLgpIi7qsKQx4NuZ+RngCuDWpp6pOjR04TZg+4Ttu4F7m9r2Aus6qWrgB8BPMvPTwGcZ1NmLcxcRy4FvAKsz82JgHnAjlfOXmZ3+AJ8Hfjph+07gzq7rmlDPk8BXgB3AsmZsGbCjo3pWMPjPtwZ4msH3db8HzJ/sfM5wbYuB12k+dJow3pdzN94U4CwG3yn4NPDnlfPX+YrDCXQ4mGkRcT5wCbCFqTs0zLT7gO8AHzfbZwMfZOZYs93l+bsAeBf4UfNW8sGIOJWenLvMfAv4PoNvn90NfAi8QOH89SE4x93hYCZFxGnAj4FvZua+rusBiIhrgT2Z+cLE4UmmdnX+5gOXAvdn5iUM7j/s9Jp1oubaai2wEvgUcCqDS4QjHfP89SE4vetwEBGfYBCaTZn5RDM8VYeGmfRF4KsR8VvgEQZv1+5j0MBr/OuMuzx/I8BIZm5pth9nEKQ+nDuALwOvZ+a7mfl74AngCxTOXx+C8zywqvlk42QGF2tPdVVMRATwELA9M++ZsGu8QwMc3qFhxmTmnZm5IjPPZ3Cens3Mm4HngOu7rK2p723gzYi4sBm6CniFHpy7xhvAFRFxSvPvPF7fiZ+/Li7SJrlouwb4DfC/wD90XMufMliqfwlsa36uYXAtsRl4tXk8q+M6vwQ83fx+AfA/DLpE/DvwyQ7r+hNga3P+/gM4s0/nDvhH4NfAS8C/AZ+snD9vuZEK+vBWTZp1DI5UYHCkAoMjFRgcqcDgSAUGRyr4PxKPPxH6oIJEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(110, 84)\n"
     ]
    }
   ],
   "source": [
    "resized = cv2.resize(gray, (84,110))\n",
    "plt.imshow(resized, cmap=\"gray\")\n",
    "plt.show()\n",
    "print(resized.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cropping the image\n",
    "\n",
    "Cropping the image to 84×84"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAANKklEQVR4nO3df+hd9X3H8edr+aHVzsS4RjIji0JIlYHRBadzjC7Wzbmi+6MdShllCP2n29QWWt3+KIX9YWFUi4yCaDsZzh+1tpVQ7EJiGYOR+iNZq0ZrtJlm2sR1JnYJ6NK+98c9sV+yb8z55n7v997r5/mAy73nc+7lfA6H1/dz7rnn+3mnqpD03vcr4+6ApIVh2KVGGHapEYZdaoRhlxph2KVGDBX2JFcmeT7JriQ3z1enJM2/nOjv7EkWAT8CrgD2AI8D11XVs/PXPUnzZfEQn70Y2FVVLwEkuR+4Bjhm2JctW1YrV64cYpOS3s2+ffs4cOBAZls3TNjPAl6ZsbwH+O13+8DKlSu57bbbhtikpHdz0003HXPdMN/ZZ/vr8f++EyT5ZJInkjxx4MCBITYnaRjDhH0PcPaM5dXAq0e/qarurKoNVbVh2bJlQ2xO0jCGCfvjwNok5yRZClwLPDI/3ZI03074O3tVHU7yF8B3gUXAV6vqmXnrmaR5NcwFOqrqO8B35qkvkkbIO+ikRhh2qRGGXWqEYZcaYdilRhh2qRFjC/vWrVvZunXruDYvNceRXWqEYZcaMdQddMM48q+uGzduHFcXpKY4skuNGNvIvmbNmnFtWmqSI7vUCMMuNWJsp/F33HHHuDYtNcmRXWqEYZcacdywJ/lqkn1Jnp7RtiLJ5iQvdM+nj7abkobVZ2T/B+DKo9puBrZU1VpgS7csaYIdN+xV9S/Afx/VfA1wT/f6HuBP5rlfkubZiX5nP7OqXgPonq3pJE24kV+gsyKMNBlONOx7k6wC6J73HeuNVoSRJsOJhv0R4BPd608A356f7kgalT4/vd0H/BuwLsmeJNcDtwJXJHmBQX32W0fbTUnDOu7tslV13TFWXT7PfZE0Qt5BJzXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjXCsEuNMOxSIwy71AjDLjWiz7RUZyd5LMnOJM8kuaFrtyqMNEX6jOyHgc9U1XnAJcCnkpyPVWGkqdKnIsxrVfVU9/pnwE7gLKwKI02VOX1nT7IGuBDYRs+qMBaJkCZD77AneT/wDeDGqnqz7+csEiFNhl5hT7KEQdDvraqHu+beVWEkjV+fq/EB7gZ2VtWXZqyyKow0RY5bJAK4DPgz4IdJdnRtf82gCsyDXYWYl4GPjaaLkuZDn4ow/wrkGKutCiNNCe+gkxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxph2KVGGHapEYZdaoRhlxrRZw66k5N8P8m/dxVhvtC1n5NkW1cR5oEkS0ffXUknqs/I/hawsaouANYDVya5BPgicFtXEeYN4PrRdVPSsPpUhKmq+p9ucUn3KGAj8FDXbkUYacL1nTd+UTez7D5gM/AisL+qDndv2cOgJNRsn7UijDQBeoW9qn5eVeuB1cDFwHmzve0Yn7UijDQB5nQ1vqr2A99jUM11eZIjU1GvBl6d365Jmk99rsZ/IMny7vX7gA8zqOT6GPDR7m1WhJEmXJ+KMKuAe5IsYvDH4cGq2pTkWeD+JH8LbGdQIkrSDGvWrHnn9bp16955vXfvXgB27Nhx9EdGpk9FmB8wKNN8dPtLDL6/S5oC3kEnNaLPabykE7R06S9vLF2yZMkYe+LILjXDkV0ag5NOOmnBt+nILjXCsEuN8DReGoO33nprwbfpyC41wpFdGgMv0EkaGcMuNcKwS40w7FIjDLvUCK/GS2Pg7+ySRsaRXRqDif6dvZtOenuSTd2yFWGkKTKX0/gbGEw0eYQVYaQp0rdIxGrgj4G7uuVgRRjpuN5+++13Hvv373/ncfDgQQ4ePLigfek7st8OfBb4Rbd8BlaEkabKcS/QJfkIsK+qnkzyoSPNs7z1mBVhgDsB1q5dO+t7pPeq3bt3z/p6HPpcjb8MuDrJVcDJwGkMRvrlSRZ3o7sVYaQJ16eK6y1Vtbqq1gDXAlur6uNYEUaaKsPcVPM54NNJdjH4Dm9FGGmCzemmmqr6HoPCjlaEkaaMt8tKjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjeg1U02S3cDPgJ8Dh6tqQ5IVwAPAGmA38KdV9cZouilpWHMZ2X+/qtZX1YZu+WZgS1cRZku3LGlCDXMafw2DSjBgRRhp4vUNewH/nOTJJJ/s2s6sqtcAuueVs33QijDSZOg7u+xlVfVqkpXA5iTP9d2AFWGkydBrZK+qV7vnfcA3GUwhvTfJKoDued+oOilpeMcNe5JTk/zqkdfAHwBPA48wqAQDVoSRJl6f0/gzgW8OqjSzGPinqno0yePAg0muB14GPja6bkoa1nHD3lV+uWCW9p8Cl4+iU5Lmn3fQSY0w7FIjDLvUCMMuNcKwS40w7FIjDLvUCMMuNcKwS40w7FIjDLvUCMMuNcKwS40w7FIjDLvUCMMuNcKwS43oFfYky5M8lOS5JDuTXJpkRZLNSV7onk8fdWclnbi+I/uXgUer6oMMpqjaiRVhpKnSZ3bZ04DfA+4GqKq3q2o/VoSRpkqfkf1c4HXga0m2J7mrm1LaijDSFOkT9sXARcBXqupC4CBzOGWvqjurakNVbVi2bNkJdlPSsPqEfQ+wp6q2dcsPMQi/FWGkKXLcsFfVT4BXkqzrmi4HnsWKMNJU6VvY8S+Be5MsBV4C/pzBHworwkhTolfYq2oHsGGWVVaEkaaEd9BJjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41wrBLjTDsUiMMu9QIwy41os9U0uuS7JjxeDPJjRaJkKZLnznonq+q9VW1Hvgt4BDwTSwSIU2VuZ7GXw68WFX/gUUipKky17BfC9zXve5VJELSZOgd9m5m2auBr89lA1aEkSbDXEb2PwKeqqq93XKvIhFWhJEmw1zCfh2/PIUHi0RIU6VvffZTgCuAh2c03wpckeSFbt2t8989SfOlb5GIQ8AZR7X9FItESFPDO+ikRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRhh2qRGGXWqEYZcaYdilRvSdluqmJM8keTrJfUlOTnJOkm1dRZgHutlnJU2oPuWfzgL+CthQVb8JLGIwf/wXgdu6ijBvANePsqOShtP3NH4x8L4ki4FTgNeAjcBD3XorwkgTrk+tt/8E/g54mUHIDwBPAvur6nD3tj3AWaPqpKTh9TmNP51BXbdzgF8HTmVQMOJodYzPWxFGmgB9TuM/DPy4ql6vqv9lMHf87wDLu9N6gNXAq7N92Iow0mToE/aXgUuSnJIkDOaKfxZ4DPho9x4rwkgTrs939m0MLsQ9Bfyw+8ydwOeATyfZxaCAxN0j7KekIfWtCPN54PNHNb8EXDzvPZI0Et5BJzWi18g+Xw4dOsT27dsXcpNSUw4dOnTMdY7sUiMMu9SIVM16L8xoNpa8DhwE/mvBNjp6v4b7M6neS/sC/fbnN6rqA7OtWNCwAyR5oqo2LOhGR8j9mVzvpX2B4ffH03ipEYZdasQ4wn7nGLY5Su7P5Hov7QsMuT8L/p1d0nh4Gi81YkHDnuTKJM8n2ZXk5oXc9rCSnJ3ksSQ7u/n4bujaVyTZ3M3Ft7n7//+pkWRRku1JNnXLUzu3YJLlSR5K8lx3nC6d5uMz33M/LljYkywC/p7BxBfnA9clOX+htj8PDgOfqarzgEuAT3X9vxnY0s3Ft6VbniY3ADtnLE/z3IJfBh6tqg8CFzDYr6k8PiOZ+7GqFuQBXAp8d8byLcAtC7X9EezPt4ErgOeBVV3bKuD5cfdtDvuwmkEANgKbgDC4aWPxbMdskh/AacCP6a5DzWifyuPDYJq3V4AVDP6HZRPwh8Mcn4U8jT/S+SOmdt66JGuAC4FtwJlV9RpA97xyfD2bs9uBzwK/6JbPYHrnFjwXeB34Wve15K4kpzKlx6dGMPfjQoY9s7RN3U8BSd4PfAO4sareHHd/TlSSjwD7qurJmc2zvHVajtFi4CLgK1V1IYPbsqfilH02w879OJuFDPse4OwZy8ect25SJVnCIOj3VtXDXfPeJKu69auAfePq3xxdBlydZDdwP4NT+dvpObfgBNoD7KnBzEowmF3pIqb3+Aw19+NsFjLsjwNru6uJSxlcbHhkAbc/lG7+vbuBnVX1pRmrHmEwBx9M0Vx8VXVLVa2uqjUMjsXWqvo4Uzq3YFX9BHglybqu6chciVN5fBjF3I8LfNHhKuBHwIvA34z7Isgc+/67DE6ZfgDs6B5XMfieuwV4oXteMe6+nsC+fQjY1L0+F/g+sAv4OnDSuPs3h/1YDzzRHaNvAadP8/EBvgA8BzwN/CNw0jDHxzvopEZ4B53UCMMuNcKwS40w7FIjDLvUCMMuNcKwS40w7FIj/g/1QvkjJvPxigAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84)\n"
     ]
    }
   ],
   "source": [
    "cropped = resized[18:102,:]\n",
    "plt.imshow(cropped, cmap=\"gray\")\n",
    "plt.show()\n",
    "print(cropped.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking the for last screens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84, 84)\n",
      "(1, 84, 84)\n"
     ]
    }
   ],
   "source": [
    "print(cropped.shape)\n",
    "cropped = cropped.reshape((1,84,84))\n",
    "print(cropped.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4, 84, 84)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stacked = np.vstack([cropped, cropped, cropped, cropped])\n",
    "stacked.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adding new state \n",
    "new_state = np.ones((1,84,84))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 84, 84)\n"
     ]
    }
   ],
   "source": [
    "added = stacked[:3,:,:]\n",
    "added = np.concatenate((new_state, added))\n",
    "print(added.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building the CNN for the DQN\n",
    "\n",
    "Building the original CNN of the DQN Paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# building the convolution layers for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn_1 = nn.Conv2d(4, out_channels=32, kernel_size=8, stride=4)\n",
    "cnn_2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=4, stride=2)\n",
    "cnn_3 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "img = torch.from_numpy(stacked).float().unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape first layer:  torch.Size([1, 32, 20, 20])\n",
      "shape second_layer:  torch.Size([1, 64, 9, 9])\n",
      "shape third_layer:  torch.Size([1, 64, 7, 7])\n"
     ]
    }
   ],
   "source": [
    "x = cnn_1(img)\n",
    "print(\"shape first layer: \", x.shape)\n",
    "x = cnn_2(x)\n",
    "print(\"shape second_layer: \", x.shape)\n",
    "x = cnn_3(x)\n",
    "print(\"shape third_layer: \", x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_shape: 3136\n"
     ]
    }
   ],
   "source": [
    "flattened_input = x.flatten()\n",
    "input_shape = x.flatten().shape[0]\n",
    "print(\"input_shape:\", input_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "ff_1 = nn.Linear(input_shape, 512)\n",
    "ff_2 = nn.Linear(512, env.action_space.n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building the final DQN class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self, action_size, input_shape = (4,84,84)):\n",
    "        super(DQN, self).__init__()\n",
    "        self.input_shape = input_shape\n",
    "        self.action_size = action_size\n",
    "        self.cnn_1 = nn.Conv2d(4, out_channels=32, kernel_size=8, stride=4)\n",
    "        self.cnn_2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=4, stride=2)\n",
    "        self.cnn_3 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1)\n",
    "        \n",
    "        self.ff_1 = nn.Linear(self.calc_input_layer(), 512)\n",
    "        self.ff_2 = nn.Linear(512, action_size)\n",
    "        \n",
    "    def calc_input_layer(self):\n",
    "        x = torch.zeros(self.input_shape).unsqueeze(0)\n",
    "        x = self.cnn_1(x)\n",
    "        x = self.cnn_2(x)\n",
    "        x = self.cnn_3(x)\n",
    "        return x.flatten().shape[0]\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        \n",
    "        \"\"\"\n",
    "        x = torch.relu(self.cnn_1(x))\n",
    "        x = torch.relu(self.cnn_2(x))\n",
    "        x = torch.relu(self.cnn_3(x))\n",
    "        x = x.flatten()\n",
    "        x = torch.relu(self.ff_1(x))\n",
    "        out = self.ff_2(x)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "dqn = DQN(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 4.2489, -3.7532,  0.4237, -4.3808], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dqn(img)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
